require 'optparse'
require 'sqlite3'
require 'rubygems'
require 'active_record'
require 'mechanize'

$options = {}
parser = OptionParser.new("", 24) do |opts|
  opts.banner = "\nScraper 1.0\nAuthor: Louis (Skype: louisprm)\n\n"
  
  opts.on("-i", "--input FILE", "Input") do |v|
    $options[:input] = v
  end

  opts.on("-o", "--output SQLITE3DB", "Output SQLite3 database file") do |v|
    $options[:output] = v
  end

  opts.on("-n", "--new", "Start new, no resume") do |v|
    $options[:new] = v
  end

  opts.on_tail('-h', '--help', 'Displays this help') do
		puts opts, "", help
    exit
	end
end

def help
  return <<-eos

GUIDELINE
-------------------------------------------------------
The scraper package includes two scripts

  1. scrape.rb: scrape data from the internet and store to a local database file
  2. export.rb: read the local database and generate the Excel/CSV output

Procedures:

  1. Run the scrape script and store scraped data to local database file main.db
	   
        ruby scrape.rb --output=main.db

  2. After the scraper script is done, run the export.rb script to read the main.db
     database and generate the Excel file data.xls

        ruby export.rb --input=main.db --output=/tmp/data.xls

Notes:

- The scrape.rb script supports resuming. Just run the script over and over again
  in case of any failure (due to internet connection problem for instance)to have
  it start from where it left off. Be sure to specify the same output database file
- As the scrape script stores items ony-by-one, you can run the export script
  even when the scraping process is not complete yet. Then it will export available
  items in the local database
 
eos
end

begin
  parser.parse!
rescue SystemExit => ex
  exit
rescue Exception => ex
  puts "\nERROR: #{ex.message}\n\nRun ruby crawler.rb -h for help\n\n"
  exit
end

if $options[:output].nil?
  puts "\nPlease specify output file: -o\n\n"
  exit
end

if $options[:input].nil?
  puts "\nPlease specify input file: -i\n\n"
  exit
end

ActiveRecord::Base.establish_connection(
  adapter: 'sqlite3',
  database: $options[:output],
  timeout: 15000
)

require 'zlib'
class String
  def deflate
    Zlib.deflate(self)
  end

  def inflate
    Zlib.inflate(self)
  end

  def fix
    self.encode!('UTF-8', :undef => :replace, :invalid => :replace, :replace => "")
  end
end

class MySchema < ActiveRecord::Migration
  def change
    create_table :items do |t|
      t.string :domain
      t.string :contact_name
      t.string :contact_phone
      t.string :contact_email
      t.string :status
    end
  end
end

# initiate the database if not existed
MySchema.new.migrate(:change) unless File.exists?($options[:output])

class Item < ActiveRecord::Base
end

class Scrape
  SITE = 'http://abtec.co.nz/'
  REGEXP = /[\(\+]{0,2}[0-9\.\-\s\(\)]{6,20}/

  def initialize
    @a = Mechanize.new
    @a.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
  end

  def run
    p $options
    domains = IO.read($options[:input]).split(/[\r\n]+/).compact.map{|e| e.split(",").first }.compact
    domains.each do |domain|
      get(domain)
    end
  end

  def get(domain)
    p domain
    domain = 'http://' + domain unless domain[/^http/]

    if Item.exists?(domain: domain)
      puts "Already scraped"
      puts "--------------------------------------"
      return
    end
    
    i = Item.new
    i.domain = domain

    begin
      Timeout::timeout(30) {
        ps = @a.get(File.join(domain, 'content/contact.html')).parser
      }
    rescue Exception => ex
      i.status = 'failed'
      i.save
      p  "failed"
      return
    end
    
    i.contact_name = ps.css('input[type=hidden][name=leadName]').first.attributes['value'].value if ps.css('input[type=hidden][name=leadName]').first
    i.contact_email = ps.css('input[type=hidden][name=accountEmail]').first.attributes['value'].value if ps.css('input[type=hidden][name=accountEmail]').first
    i.contact_phone = ps.css('span.number').first.text.strip if ps.css('span.number').first
    
    p 'no phone 1' unless i.contact_phone
    
    begin
      i.contact_phone = ps.css('body').first.inner_html.split(/[\r\n]+/).select{|line| line[/phone\:*/i]}.select{|line| line[/phone.*[\(\+]{0,2}[0-9\.\-\s\(\)]{6,20}/i] }.first.scan(REGEXP).select{|e| valid_phone_number?(e)}.first
    rescue Exception => ex
      
    end
    p 'no phone 2' unless i.contact_phone

    p 'no email' unless i.contact_email
    p 'no name' unless i.contact_name

    i.save
    puts "Done"
    puts "--------------------------------------"
  end

  def valid_phone_number?(str)
    str = str.strip.gsub(/^\(/, '').gsub(/\)$/, '')

    return if str.nil? or str.empty?
    
    matched = true

    exclude_patterns = [ /[\.\-\(\)]{2}/   ,  /[0-9]+\(/ ] 

    exclude_patterns.each do |ex|
      matched = false if ex =~ str
    end

    # if size < 10
    matched = false if str.gsub(/[^0-9]/, "").size < 10
    # if size > 12
    matched = false if str.gsub(/[^0-9]/, "").size > 11
    # if size > 12
    matched = false if str.gsub(/[^0-9]/, "").size == 11 and str.gsub(/[^0-9]/, "")[0] != '1'
    # if more than 4 dots
    matched = false if str.gsub(/[^\.]/, "").size > 3
    # if more than 4 dashes
    matched = false if str.gsub(/[^\-]/, "").size > 3
    # if more than 1 (
    matched = false if str.gsub(/[^\(]/, "").size > 1
    # if more than 1 )
    matched = false if str.gsub(/[^\)]/, "").size > 1
    # if only digit, no other sign
    matched = false if str[/[\.\-\s\(\)]/].nil?
    # if not starting with ( + or number
    matched = false if str[/^[^0-9\+\(]/]
    # if ()ddddddddd -> d count > 10
    matched = false if str.gsub(/\+*\(.*\)/, '').gsub(/[^0-9]+/, '').size < 10
    matched = false if str.gsub(/\+*\(.*\)/, '').gsub(/[^0-9]+/, '').size > 11

    #puts "#{str} --> #{matched}"
    return matched
  end

  
end

# trap Ctrl-C
trap("SIGINT") { throw :ctrl_c }

catch :ctrl_c do
  e = Scrape.new
  e.run
end
